<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>You Have an Accent | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="token-normalization.html" title="Normalizing Tokens">
<link rel="prev" href="lowercase-token-filter.html" title="In That Case">
<link rel="next" href="unicode-normalization.html" title="Living in a Unicode World">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="languages.html">Dealing with Human Language</a></span>
»
<span class="breadcrumb-link"><a href="token-normalization.html">Normalizing Tokens</a></span>
»
<span class="breadcrumb-node">You Have an Accent</span>
</div>
<div class="navheader">
<span class="prev">
<a href="lowercase-token-filter.html">« In That Case</a>
</span>
<span class="next">
<a href="unicode-normalization.html">Living in a Unicode World »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="asciifolding-token-filter"></a>You Have an Accent<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/220_Token_normalization/20_Removing_diacritics.asciidoc">edit</a>
</h2>
</div></div></div>
<p>English uses diacritics (like <code class="literal">´</code>, <code class="literal">^</code>, and <code class="literal">¨</code>) only for imported words—​like <code class="literal">rôle</code>, <code class="literal">déjà</code>, and <code class="literal">däis</code>—but usually they are optional.  Other
languages require diacritics in order to be correct.  Of course, just because
words are spelled correctly in your index doesn’t mean that the user will
search for the correct spelling.</p>
<p>It is often useful to strip diacritics from words, allowing <code class="literal">rôle</code> to match
<code class="literal">role</code>, and vice versa. With Western languages, this can be done with the
<code class="literal">asciifolding</code> character filter.  Actually, it does more than just strip
diacritics.  It tries to convert many Unicode characters into a simpler ASCII
representation:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<code class="literal">ß</code> ⇒ <code class="literal">ss</code>
</li>
<li class="listitem">
<code class="literal">æ</code> ⇒ <code class="literal">ae</code>
</li>
<li class="listitem">
<code class="literal">ł</code> ⇒ <code class="literal">l</code>
</li>
<li class="listitem">
<code class="literal">ɰ</code> ⇒ <code class="literal">m</code>
</li>
<li class="listitem">
<code class="literal">⁇</code> ⇒ <code class="literal">??</code>
</li>
<li class="listitem">
<code class="literal">❷</code> ⇒ <code class="literal">2</code>
</li>
<li class="listitem">
<code class="literal">⁶</code> ⇒ <code class="literal">6</code>
</li>
</ul>
</div>
<p>Like the <code class="literal">lowercase</code> filter, the <code class="literal">asciifolding</code> filter doesn’t require any
configuration but can be included directly in a <code class="literal">custom</code> analyzer:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT /my_index
{
  "settings": {
    "analysis": {
      "analyzer": {
        "folding": {
          "tokenizer": "standard",
          "filter":  [ "lowercase", "asciifolding" ]
        }
      }
    }
  }
}

GET /my_index?analyzer=folding
My œsophagus caused a débâcle <a id="CO140-1"></a><i class="conum" data-value="1"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO140-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Emits <code class="literal">my</code>, <code class="literal">oesophagus</code>, <code class="literal">caused</code>, <code class="literal">a</code>, <code class="literal">debacle</code></p>
</td>
</tr>
</table>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_retaining_meaning"></a>Retaining Meaning<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/220_Token_normalization/20_Removing_diacritics.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Of course, when you strip diacritical marks from a word, you lose meaning.
For instance, consider these three Spanish words:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<code class="literal">esta</code>
</span>
</dt>
<dd>
Feminine form of the adjective <em>this</em>, as in <em>esta silla</em> (this chair) or <em>esta</em> (this one).
</dd>
<dt>
<span class="term">
<code class="literal">ésta</code>
</span>
</dt>
<dd>
An archaic form of <code class="literal">esta</code>.
</dd>
<dt>
<span class="term">
<code class="literal">está</code>
</span>
</dt>
<dd>
The third-person form of the verb <em>estar</em> (to be), as in <em>está feliz</em> (he is happy).
</dd>
</dl>
</div>
<p>While we would like to conflate the first two forms, they differ in meaning
from the third form, which we would like to keep separate.  Similarly:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<code class="literal">sé</code>
</span>
</dt>
<dd>
The first person form of the verb <em>saber</em> (to know) as in <em>Yo sé</em>  (I know).
</dd>
<dt>
<span class="term">
<code class="literal">se</code>
</span>
</dt>
<dd>
The third-person reflexive pronoun used with many verbs, such as <em>se sabe</em> (it is known).
</dd>
</dl>
</div>
<p>Unfortunately, there is no easy way to separate words that should have
their diacritics removed from words that shouldn’t.  And it is quite likely
that your users won’t know either.</p>
<p>Instead, we index the text twice: once in the original form and once with
diacritics removed:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT /my_index/_mapping/my_type
{
  "properties": {
    "title": { <a id="CO141-1"></a><i class="conum" data-value="1"></i>
      "type":           "string",
      "analyzer":       "standard",
      "fields": {
        "folded": { <a id="CO141-2"></a><i class="conum" data-value="2"></i>
          "type":       "string",
          "analyzer":   "folding"
        }
      }
    }
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO141-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">title</code> field uses the <code class="literal">standard</code> analyzer and will contain
the original word with diacritics in place.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO141-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">title.folded</code> field uses the <code class="literal">folding</code> analyzer, which strips
the diacritical marks.</p>
</td>
</tr>
</table>
</div>
<p>You can test the field mappings by using the <code class="literal">analyze</code> API on the sentence
<em>Esta está loca</em> (This woman is crazy):</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">GET /my_index/_analyze?field=title <a id="CO142-1"></a><i class="conum" data-value="1"></i>
Esta está loca

GET /my_index/_analyze?field=title.folded <a id="CO142-2"></a><i class="conum" data-value="2"></i>
Esta está loca</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO142-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Emits <code class="literal">esta</code>, <code class="literal">está</code>, <code class="literal">loca</code></p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO142-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Emits <code class="literal">esta</code>, <code class="literal">esta</code>, <code class="literal">loca</code></p>
</td>
</tr>
</table>
</div>
<p>Let’s index some documents to test it out:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT /my_index/my_type/1
{ "title": "Esta loca!" }

PUT /my_index/my_type/2
{ "title": "Está loca!" }</pre>
</div>
<p>Now we can search across both fields, using the <code class="literal">multi_match</code> query in
<a class="xref" href="most-fields.html" title="Most Fields"><code class="literal">most_fields</code> mode</a> to combine the scores from each field:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">GET /my_index/_search
{
  "query": {
    "multi_match": {
      "type":     "most_fields",
      "query":    "esta loca",
      "fields": [ "title", "title.folded" ]
    }
  }
}</pre>
</div>
<p>Running this query through the <code class="literal">validate-query</code> API helps to explain how the
query is executed:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">GET /my_index/_validate/query?explain
{
  "query": {
    "multi_match": {
      "type":     "most_fields",
      "query":    "está loca",
      "fields": [ "title", "title.folded" ]
    }
  }
}</pre>
</div>
<p>The <code class="literal">multi-match</code> query searches for the original form of the word (<code class="literal">está</code>) in the <code class="literal">title</code> field,
and the form without diacritics <code class="literal">esta</code> in the <code class="literal">title.folded</code> field:</p>
<pre class="literallayout">(title:está        title:loca       )
(title.folded:esta title.folded:loca)</pre>

<p>It doesn’t matter whether the user searches for <code class="literal">esta</code> or <code class="literal">está</code>; both
documents will match because the form without diacritics exists in the the
<code class="literal">title.folded</code> field.  However, only the original form exists in the <code class="literal">title</code>
field. This extra match will push the document containing the original form of
the word to the top of the results list.</p>
<p>We use the <code class="literal">title.folded</code> field to  <em>widen the net</em> in order to match more
documents, and use the original <code class="literal">title</code> field to push the most relevant
document to the top. This same technique can be used wherever an analyzer is
used, to increase matches at the expense of meaning.</p>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>The <code class="literal">asciifolding</code> filter does have an option called <code class="literal">preserve_original</code> that
allows you to index the original token and the folded token in the same
position in the same field.  With this option enabled, you would end up with
something like this:</p>
<pre class="literallayout">Position 1     Position 2
--------------------------
(ésta,esta)    loca
--------------------------</pre>

<p>While this appears to be a nice way to save space, it does mean that you have
no way of saying, “Give me an exact match on the original word.”  Mixing
tokens with and without diacritics can also end up interfering with term-frequency counts, resulting in less-reliable relevance calcuations.</p>
<p>As a rule, it is cleaner to index each field variant into a separate field,
as we have done in this section.</p>
</div>
</div>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="lowercase-token-filter.html">« In That Case</a>
</span>
<span class="next">
<a href="unicode-normalization.html">Living in a Unicode World »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
